package cn.whuc.spark.operator

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

object Demo_aggregateByKey {
  def main(args: Array[String]): Unit = {
    // 1 创建sparkContext
    val sc: SparkContext = new SparkContext(
      new SparkConf()
        .setMaster("local[*]")
        .setAppName(" ")
    )

    // 2 编写代码
    val rdd1: RDD[Int] = sc.makeRDD(List(1, 2, 3, 4), 2)

    // 每个分区内最大值求和
    val rdd2: RDD[(String, Int)] = rdd1.mapPartitionsWithIndex((index, iter) => {
      iter.map(("分区间最大值求和", _))
    })

    val rdd3: RDD[(String, Int)] = rdd2.aggregateByKey(0)(
      (x, y) => math.max(x, y),
      (x, y) => x + y
    )

    rdd3.collect().foreach(println)

    // 3 关闭上下文对象
    sc.stop()
  }
}
